<!DOCTYPE html><html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en" data-whc_version="25.0">
    <head><link rel="shortcut icon" href="../../../oxygen-webhelp/template/images/favicon.png"/><link rel="icon" href="../../../oxygen-webhelp/template/images/favicon.png"/><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/><meta name="viewport" content="width=device-width, initial-scale=1.0"/><meta http-equiv="X-UA-Compatible" content="IE=edge"/><meta name="copyright" content="(C) Copyright 2024"/><meta name="generator" content="DITA-OT"/><meta name="description" content="This solution describes how to design a pipeline that stops automatically after it finishes processing all available data. Let's say that your dataflow topology updates a database table daily at 4 am. ..."/><meta name="prodname" content="Data Collector"/><meta name="version" content="3"/><meta name="release" content="16"/><meta name="modification" content="0"/>        
      <title>Stopping a Pipeline After Processing All Available Data</title><!--  Generated with Oxygen version 25.1, build number 2023042410.  --><meta name="wh-path2root" content="../../../"/><meta name="wh-toc-id" content="concept_kff_ykv_lz-d16893e60248"/><meta name="wh-source-relpath" content="datacollector/UserGuide/Solutions/StopPipeline.dita"/><meta name="wh-out-relpath" content="datacollector/UserGuide/Solutions/StopPipeline.html"/>

    <link rel="stylesheet" type="text/css" href="../../../oxygen-webhelp/app/commons.css?buildId=2023042410"/>
    <link rel="stylesheet" type="text/css" href="../../../oxygen-webhelp/app/topic.css?buildId=2023042410"/>

    <script src="../../../oxygen-webhelp/app/options/properties.js?buildId=20240802104629"></script>
    <script src="../../../oxygen-webhelp/app/localization/strings.js?buildId=2023042410"></script>
    <script src="../../../oxygen-webhelp/app/search/index/keywords.js?buildId=20240802104629"></script>
    <script defer="defer" src="../../../oxygen-webhelp/app/commons.js?buildId=2023042410"></script>
    <script defer="defer" src="../../../oxygen-webhelp/app/topic.js?buildId=2023042410"></script>
<link rel="stylesheet" type="text/css" href="../../../oxygen-webhelp/template/light.css?buildId=2023042410"/><link rel="stylesheet" type="text/css" href="../../../skin.css"/></head>

    <body class="wh_topic_page frmBody">
        
        
        

        
<nav class="navbar navbar-default wh_header" data-whc_version="25.0">
    <div class="container-fluid">
        <div class="wh_header_flex_container navbar-nav navbar-expand-md navbar-dark">
            <div class="wh_logo_and_publication_title_container">
                <div class="wh_logo_and_publication_title">
                    
                    <!--
                            This component will be generated when the next parameters are specified in the transformation scenario:
                            'webhelp.logo.image' and 'webhelp.logo.image.target.url'.
                            See: http://oxygenxml.com/doc/versions/17.1/ug-editor/#topics/dita_webhelp_output.html.
                    -->
                    
                    <div class=" wh_publication_title "><a href="../../../index.html"><span class="booktitle">  <span class="ph mainbooktitle"><span class="ph">Data Collector</span> User Guide</span>  </span></a></div>
                    
                </div>
                
                <!-- The menu button for mobile devices is copied in the output only when the 'webhelp.show.top.menu' parameter is set to 'yes' -->
                
            </div>

            <div class="wh_top_menu_and_indexterms_link collapse navbar-collapse">
                
                
                <div class=" wh_indexterms_link "><a href="../../../indexTerms.html" title="Index" aria-label="Go to index terms page"><span>Index</span></a></div>
                
            </div>
        </div>
    </div>
</nav>

        <div class=" wh_search_input navbar-form wh_topic_page_search search " role="form">


<form id="searchForm" method="get" role="search" action="../../../search.html"><div><input type="search" placeholder="Search " class="wh_search_textfield" id="textToSearch" name="searchQuery" aria-label="Search query" required="required"/><button type="submit" class="wh_search_button" aria-label="Search"><span class="search_input_text">Search</span></button></div></form>

</div>
        
        <div class="container-fluid">
            <div class="row">

                <nav class="wh_tools d-print-none">
                    
<div data-tooltip-position="bottom" class=" wh_breadcrumb "><ol class="d-print-none"><li><span class="home"><a href="../../../index.html"><span>Home</span></a></span></li><li><div class="topicref" data-id="concept_zq5_pb4_flb"><div class="title"><a href="../../../datacollector/UserGuide/Solutions/Solutions-title.html">Solutions</a></div></div></li><li class="active"><div class="topicref" data-id="concept_kff_ykv_lz"><div class="title"><a href="../../../datacollector/UserGuide/Solutions/StopPipeline.html#concept_kff_ykv_lz">Stopping a Pipeline After Processing All Available Data</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li></ol></div>



                    <div class="wh_right_tools "><button class="wh_hide_highlight" aria-label="Toggle search highlights" title="Toggle search highlights"></button><button class="webhelp_expand_collapse_sections" data-next-state="collapsed" aria-label="Collapse sections" title="Collapse sections"></button><div class=" wh_navigation_links "><span id="topic_navigation_links" class="navheader">
  
<span class="navprev"><a class="- topic/link link" href="../../../datacollector/UserGuide/Solutions/FileManagement.html#concept_d1q_xl4_lx" title="Managing Output Files" aria-label="Previous topic: Managing Output Files" rel="prev"></a></span>  
<span class="navnext"><a class="- topic/link link" href="../../../datacollector/UserGuide/Solutions/SqoopReplacement.html#concept_vrh_jrs_bbb" title="Offloading Data from Relational Sources to Hadoop" aria-label="Next topic: Offloading Data from Relational Sources to Hadoop" rel="next"></a></span>  </span></div>
<!--External resource link-->
<div class=" wh_print_link print d-none d-md-inline-block "><button onClick="window.print()" title="Print this page" aria-label="Print this page"></button></div>
                        
                        
                        
                        
                    </div>
                </nav>
            </div>

            

<div class="wh_content_area">
                <div class="row">
                    


                        <nav role="navigation" id="wh_publication_toc" class="col-lg-3 col-md-3 col-sm-12 d-md-block d-none d-print-none">
<div id="wh_publication_toc_content">


                            <div class=" wh_publication_toc " data-tooltip-position="right"><span class="expand-button-action-labels"><span id="button-expand-action" role="button" aria-label="Expand"></span><span id="button-collapse-action" role="button" aria-label="Collapse"></span><span id="button-pending-action" role="button" aria-label="Pending"></span></span><ul role="tree" aria-label="Table of Contents"><li role="treeitem" aria-expanded="false"><div data-tocid="concept_htw_ghg_jq-d16893e53" class="topicref" data-id="concept_htw_ghg_jq" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_htw_ghg_jq-d16893e53-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Getting_Started/GettingStarted_Title.html#concept_htw_ghg_jq" id="concept_htw_ghg_jq-d16893e53-link">Getting Started</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_l2v_nlp_mpb-d16893e331" class="topicref" data-id="concept_l2v_nlp_mpb" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_l2v_nlp_mpb-d16893e331-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/ReleaseNotes/ReleaseNotes.html#concept_l2v_nlp_mpb" id="concept_l2v_nlp_mpb-d16893e331-link">Release Notes</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_l4q_flb_kr-d16893e2582" class="topicref" data-id="concept_l4q_flb_kr" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_l4q_flb_kr-d16893e2582-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Installation/Install_title.html" id="concept_l4q_flb_kr-d16893e2582-link">Installation</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_ylh_yyz_ky-d16893e3984" class="topicref" data-id="concept_ylh_yyz_ky" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_ylh_yyz_ky-d16893e3984-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Configuration/Config_title.html" id="concept_ylh_yyz_ky-d16893e3984-link">Configuration</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_ejk_f1f_5v-d16893e7058" class="topicref" data-id="concept_ejk_f1f_5v" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_ejk_f1f_5v-d16893e7058-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Upgrade/Upgrade_title.html" id="concept_ejk_f1f_5v-d16893e7058-link">Upgrade</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_qsw_cjy_bt-d16893e10103" class="topicref" data-id="concept_qsw_cjy_bt" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_qsw_cjy_bt-d16893e10103-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Pipeline_Design/PipelineDesign_title.html" id="concept_qsw_cjy_bt-d16893e10103-link">Pipeline Concepts and Design</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_qn1_wn4_kq-d16893e11199" class="topicref" data-id="concept_qn1_wn4_kq" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_qn1_wn4_kq-d16893e11199-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Pipeline_Configuration/PipelineConfiguration_title.html" id="concept_qn1_wn4_kq-d16893e11199-link">Pipeline Configuration</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_hdr_gyw_41b-d16893e13057" class="topicref" data-id="concept_hdr_gyw_41b" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_hdr_gyw_41b-d16893e13057-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Data_Formats/DataFormats-Title.html" id="concept_hdr_gyw_41b-d16893e13057-link">Data Formats</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_yjl_nc5_jq-d16893e14164" class="topicref" data-id="concept_yjl_nc5_jq" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_yjl_nc5_jq-d16893e14164-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Origins/Origins_title.html" id="concept_yjl_nc5_jq-d16893e14164-link">Origins</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_yjl_nc5_jq-d16893e35197" class="topicref" data-id="concept_yjl_nc5_jq" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_yjl_nc5_jq-d16893e35197-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Processors/Processors_title.html" id="concept_yjl_nc5_jq-d16893e35197-link">Processors</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_agj_cfj_br-d16893e44037" class="topicref" data-id="concept_agj_cfj_br" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_agj_cfj_br-d16893e44037-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Destinations/Destinations-title.html" id="concept_agj_cfj_br-d16893e44037-link">Destinations</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_umc_1lk_fx-d16893e56072" class="topicref" data-id="concept_umc_1lk_fx" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_umc_1lk_fx-d16893e56072-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Executors/Executors-title.html" id="concept_umc_1lk_fx-d16893e56072-link">Executors</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_xxd_f5r_kx-d16893e59696" class="topicref" data-id="concept_xxd_f5r_kx" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_xxd_f5r_kx-d16893e59696-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Event_Handling/EventFramework-Title.html#concept_xxd_f5r_kx" id="concept_xxd_f5r_kx-d16893e59696-link">Dataflow Triggers</a></div></div></li><li role="treeitem" aria-expanded="true"><div data-tocid="concept_zq5_pb4_flb-d16893e60134" class="topicref" data-id="concept_zq5_pb4_flb" data-state="expanded"><span role="button" tabindex="0" aria-labelledby="button-collapse-action concept_zq5_pb4_flb-d16893e60134-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Solutions/Solutions-title.html" id="concept_zq5_pb4_flb-d16893e60134-link">Solutions</a></div></div><ul role="group" class="navbar-nav nav-list"><li role="treeitem"><div data-tocid="concept_aw1_p1q_plb-d16893e60156" class="topicref" data-id="concept_aw1_p1q_plb" data-state="leaf"><span role="button" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Solutions/Overview.html#concept_aw1_p1q_plb" id="concept_aw1_p1q_plb-d16893e60156-link">Solutions Overview </a></div></div></li><li role="treeitem"><div data-tocid="concept_jkm_rnz_kx-d16893e60178" class="topicref" data-id="concept_jkm_rnz_kx" data-state="leaf"><span role="button" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Solutions/Parquet.html#concept_jkm_rnz_kx" id="concept_jkm_rnz_kx-d16893e60178-link">Converting Data to the Parquet Data Format</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem"><div data-tocid="concept_szz_xwm_lx-d16893e60202" class="topicref" data-id="concept_szz_xwm_lx" data-state="leaf"><span role="button" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Solutions/Impala.html#concept_szz_xwm_lx" id="concept_szz_xwm_lx-d16893e60202-link">Automating Impala Metadata Updates for Drift Synchronization for Hive</a></div></div></li><li role="treeitem"><div data-tocid="concept_d1q_xl4_lx-d16893e60224" class="topicref" data-id="concept_d1q_xl4_lx" data-state="leaf"><span role="button" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Solutions/FileManagement.html#concept_d1q_xl4_lx" id="concept_d1q_xl4_lx-d16893e60224-link">Managing Output Files</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" class="active"><div data-tocid="concept_kff_ykv_lz-d16893e60248" class="topicref" data-id="concept_kff_ykv_lz" data-state="leaf"><span role="button" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Solutions/StopPipeline.html#concept_kff_ykv_lz" id="concept_kff_ykv_lz-d16893e60248-link">Stopping a Pipeline After Processing All Available Data</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem"><div data-tocid="concept_vrh_jrs_bbb-d16893e60272" class="topicref" data-id="concept_vrh_jrs_bbb" data-state="leaf"><span role="button" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Solutions/SqoopReplacement.html#concept_vrh_jrs_bbb" id="concept_vrh_jrs_bbb-d16893e60272-link">Offloading Data from Relational Sources to Hadoop</a></div></div></li><li role="treeitem"><div data-tocid="concept_t2t_lp5_xz-d16893e60294" class="topicref" data-id="concept_t2t_lp5_xz" data-state="leaf"><span role="button" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Solutions/SendEmail.html#concept_t2t_lp5_xz" id="concept_t2t_lp5_xz-d16893e60294-link">Sending Email During Pipeline Processing</a></div></div></li><li role="treeitem"><div data-tocid="concept_ocb_nnl_px-d16893e60316" class="topicref" data-id="concept_ocb_nnl_px" data-state="leaf"><span role="button" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Solutions/EventStorage.html#concept_ocb_nnl_px" id="concept_ocb_nnl_px-d16893e60316-link">Preserving an Audit Trail of Events</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_a5b_wvk_ckb-d16893e60338" class="topicref" data-id="concept_a5b_wvk_ckb" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_a5b_wvk_ckb-d16893e60338-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Solutions/DeltaLake.html#concept_a5b_wvk_ckb" id="concept_a5b_wvk_ckb-d16893e60338-link">Loading Data into Databricks Delta Lake</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_phk_bdf_2w-d16893e60616" class="topicref" data-id="concept_phk_bdf_2w" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_phk_bdf_2w-d16893e60616-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Solutions/HiveDrift-Overview.html#concept_phk_bdf_2w" id="concept_phk_bdf_2w-d16893e60616-link">Drift Synchronization Solution for Hive</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_ljq_knr_4cb-d16893e61090" class="topicref" data-id="concept_ljq_knr_4cb" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_ljq_knr_4cb-d16893e61090-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Solutions/JDBC_DriftSyncSolution.html#concept_ljq_knr_4cb" id="concept_ljq_knr_4cb-d16893e61090-link"><span class="ph">Drift Synchronization Solution for PostgreSQL</span></a></div></div></li></ul></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_ugp_kwf_xw-d16893e61337" class="topicref" data-id="concept_ugp_kwf_xw" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_ugp_kwf_xw-d16893e61337-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/DPM/DPM_title.html" id="concept_ugp_kwf_xw-d16893e61337-link">StreamSets Control Hub</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_fyf_gkq_4bb-d16893e62693" class="topicref" data-id="concept_fyf_gkq_4bb" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_fyf_gkq_4bb-d16893e62693-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Edge_Mode/EdgePipelines_title.html" id="concept_fyf_gkq_4bb-d16893e62693-link"><span class="ph">StreamSets Data Collector Edge</span></a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_wwq_gxc_py-d16893e63980" class="topicref" data-id="concept_wwq_gxc_py" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_wwq_gxc_py-d16893e63980-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Multithreaded_Pipelines/MultithreadedPipelines.html#concept_wwq_gxc_py" id="concept_wwq_gxc_py-d16893e63980-link">Multithreaded Pipelines</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_gzw_tdm_p2b-d16893e64187" class="topicref" data-id="concept_gzw_tdm_p2b" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_gzw_tdm_p2b-d16893e64187-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Microservice/Microservice_Title.html#concept_gzw_tdm_p2b" id="concept_gzw_tdm_p2b-d16893e64187-link">Microservice Pipelines</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="Orchestrators_Title-d16893e64348" class="topicref" data-id="Orchestrators_Title" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action Orchestrators_Title-d16893e64348-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Orchestration_Pipelines/OrchestrationPipelines_Title.html#Orchestrators_Title" id="Orchestrators_Title-d16893e64348-link">Orchestration Pipelines</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_wr1_ktz_bt-d16893e64489" class="topicref" data-id="concept_wr1_ktz_bt" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_wr1_ktz_bt-d16893e64489-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/RPC_Pipelines/SDC_RPCpipelines_title.html#concept_wr1_ktz_bt" id="concept_wr1_ktz_bt-d16893e64489-link">SDC RPC Pipelines</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_fpz_5r4_vs-d16893e64679" class="topicref" data-id="concept_fpz_5r4_vs" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_fpz_5r4_vs-d16893e64679-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Cluster_Mode/ClusterPipelines_title.html" id="concept_fpz_5r4_vs-d16893e64679-link">Cluster Pipelines</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_jjk_23z_sq-d16893e65172" class="topicref" data-id="concept_jjk_23z_sq" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_jjk_23z_sq-d16893e65172-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Data_Preview/DataPreview_Title.html#concept_jjk_23z_sq" id="concept_jjk_23z_sq-d16893e65172-link">Data Preview</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_pgk_brx_rr-d16893e65458" class="topicref" data-id="concept_pgk_brx_rr" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_pgk_brx_rr-d16893e65458-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Alerts/RulesAlerts_title.html#concept_pgk_brx_rr" id="concept_pgk_brx_rr-d16893e65458-link">Rules and Alerts</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_asx_fdz_sq-d16893e65960" class="topicref" data-id="concept_asx_fdz_sq" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_asx_fdz_sq-d16893e65960-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Pipeline_Monitoring/PipelineMonitoring_title.html#concept_asx_fdz_sq" id="concept_asx_fdz_sq-d16893e65960-link">Pipeline Monitoring</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_o3l_dtr_5q-d16893e66304" class="topicref" data-id="concept_o3l_dtr_5q" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_o3l_dtr_5q-d16893e66304-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Pipeline_Maintenance/PipelineMaintenance_title.html#concept_o3l_dtr_5q" id="concept_o3l_dtr_5q-d16893e66304-link">Pipeline Maintenance</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_yms_ftm_sq-d16893e66768" class="topicref" data-id="concept_yms_ftm_sq" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_yms_ftm_sq-d16893e66768-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Administration/Administration_title.html#concept_yms_ftm_sq" id="concept_yms_ftm_sq-d16893e66768-link">Administration</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_nls_w1r_ks-d16893e67508" class="topicref" data-id="concept_nls_w1r_ks" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_nls_w1r_ks-d16893e67508-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Tutorial/Tutorial-title.html" id="concept_nls_w1r_ks-d16893e67508-link">Tutorial</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_sh3_frm_tq-d16893e68001" class="topicref" data-id="concept_sh3_frm_tq" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_sh3_frm_tq-d16893e68001-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Troubleshooting/Troubleshooting_title.html#concept_sh3_frm_tq" id="concept_sh3_frm_tq-d16893e68001-link">Troubleshooting</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_xbx_rs1_tq-d16893e68798" class="topicref" data-id="concept_xbx_rs1_tq" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_xbx_rs1_tq-d16893e68798-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Glossary/Glossary_title.html#concept_xbx_rs1_tq" id="concept_xbx_rs1_tq-d16893e68798-link">Glossary</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_jn1_nzb_kv-d16893e68843" class="topicref" data-id="concept_jn1_nzb_kv" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_jn1_nzb_kv-d16893e68843-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Apx-DataFormats/DataFormat_Title.html#concept_jn1_nzb_kv" id="concept_jn1_nzb_kv-d16893e68843-link">Data Formats by Stage</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_pvm_yt3_wq-d16893e68958" class="topicref" data-id="concept_pvm_yt3_wq" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_pvm_yt3_wq-d16893e68958-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Expression_Language/ExpressionLanguage_title.html" id="concept_pvm_yt3_wq-d16893e68958-link">Expression Language</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_vcj_1ws_js-d16893e69669" class="topicref" data-id="concept_vcj_1ws_js" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_vcj_1ws_js-d16893e69669-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Apx-RegEx/RegEx-Title.html#concept_vcj_1ws_js" id="concept_vcj_1ws_js-d16893e69669-link">Regular Expressions</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_chv_vmj_wr-d16893e69787" class="topicref" data-id="concept_chv_vmj_wr" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_chv_vmj_wr-d16893e69787-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Apx-GrokPatterns/GrokPatterns_title.html#concept_chv_vmj_wr" id="concept_chv_vmj_wr-d16893e69787-link">Grok Patterns</a></div></div></li></ul></div>
                        

</div>
</nav>
                    


                    
                    <div id="wh_topic_body" class="col-lg-7 col-md-9 col-sm-12">
<button id="wh_close_publication_toc_button" class="close-toc-button d-none" aria-label="Toggle publishing table of content" aria-controls="wh_publication_toc" aria-expanded="true"><span class="close-toc-icon-container"><span class="close-toc-icon"></span></span></button><button id="wh_close_topic_toc_button" class="close-toc-button d-none" aria-label="Toggle topic table of content" aria-controls="wh_topic_toc" aria-expanded="true"><span class="close-toc-icon-container"><span class="close-toc-icon"></span></span></button>

                        
<div class=" wh_topic_content body "><main role="main"><article class="" role="article" aria-labelledby="ariaid-title1"><article class="nested0" aria-labelledby="ariaid-title1" id="concept_kff_ykv_lz">
    <h1 class="- topic/title title topictitle1" id="ariaid-title1">Stopping a Pipeline After Processing All Available Data</h1>
    
    <div class="- topic/body concept/conbody body conbody"><p class="- topic/shortdesc shortdesc"></p>
        <p class="- topic/p p">This solution describes how to design a pipeline that stops automatically after it
            finishes processing all available data.</p>
        <p class="- topic/p p">Let's say that your dataflow topology updates a database table daily at 4 am. Rather than
            have the pipeline process the data in a few minutes and sit idle for the rest of the
            day, you want to kick off the pipeline, have it process all data and then stop - just
            like old school batch processing. And you'd like to have the pipeline let you know when
            it has stopped. </p>
        <p class="- topic/p p">To do this, simply configure the origin to <a class="- topic/xref xref" href="../Event_Handling/EventFramework-Title.html#concept_cph_5h4_lx">generate
                events</a> and then route the no-more-data event record to the Pipeline Finisher
            executor and configure notification.</p>
        <div class="- topic/p p">The following origins generate no-more-data
                        events:<ul class="- topic/ul ul" id="concept_kff_ykv_lz__ul_l51_mvd_nx" data-ofbid="concept_kff_ykv_lz__ul_l51_mvd_nx">
                        <li class="- topic/li li">Amazon S3 origin</li>
                        <li class="- topic/li li">Azure Blob Storage origin</li>
                        <li class="- topic/li li">Azure Data Lake Storage Gen1 origin</li>
                        <li class="- topic/li li">Azure Data Lake Storage Gen2 origin</li>
                        <li class="- topic/li li">Azure Data Lake Storage Gen2 (Legacy) origin</li>
                        
                        <li class="- topic/li li">Directory origin</li>
                        <li class="- topic/li li">Google Cloud Storage origin</li>
                        <li class="- topic/li li">Hadoop FS Standalone origin</li>
                        <li class="- topic/li li">JDBC Multitable Consumer origin</li>
                        <li class="- topic/li li">JDBC Query Consumer origin</li>
                        <li class="- topic/li li">MongoDB origin</li>
                        <li class="- topic/li li">MongoDB Atlas origin</li>
                        <li class="- topic/li li">Oracle Multitable Consumer origin</li>
                        <li class="- topic/li li">Salesforce origin</li>
                        <li class="- topic/li li">Salesforce Bulk API 2.0 origin</li>
                        <li class="- topic/li li">SAP HANA Query Consumer origin</li>
                        <li class="- topic/li li">SFTP/FTP/FTPS Client origin</li>
                        <li class="- topic/li li">SQL Server 2019 BDC Multitable Consumer origin</li>
                        <li class="- topic/li li">SQL Server CDC Client origin</li>
                        <li class="- topic/li li">SQL Server Change Tracking origin</li>
                        <li class="- topic/li li">Teradata Consumer origin</li>
                        <li class="- topic/li li">Web Client origin</li>
                  </ul></div>
        <p class="- topic/p p">We'll use the JDBC Query Consumer to show a more complex scenario. </p>
        <p class="- topic/p p">Here's the basic pipeline that reads from a database, performs some processing, and
            writes to HDFS: </p>
        <p class="- topic/p p"><img class="- topic/image image" id="concept_kff_ykv_lz__image_zy1_kkw_lz" src="../Graphics/Event-StopPipe-Basic.png" height="82" width="634"/></p>
        <div class="- topic/p p">To configure the pipeline to stop after processing all available queried data:<ol class="- topic/ol ol" id="concept_kff_ykv_lz__ol_ivb_qmw_lz" data-ofbid="concept_kff_ykv_lz__ol_ivb_qmw_lz">
                <li class="- topic/li li">Configure the origin to generate events:<p class="- topic/p p">On the <span class="+ topic/keyword ui-d/wintitle keyword wintitle">General</span>
                        tab of the JDBC Query Consumer origin, select the <span class="+ topic/ph ui-d/uicontrol ph uicontrol">Produce
                            Events</span> property. </p><p class="- topic/p p">The event output stream becomes
                        available:</p><p class="- topic/p p"><img class="- topic/image image" id="concept_kff_ykv_lz__image_b5v_znw_lz" src="../Graphics/Event-StopPipe-Event.png" height="229" width="574"/></p><p class="- topic/p p">The JDBC Query Consumer
                        generates several types of events: query success, query failure, and
                        no-more-data. We know this because you checked the <a class="- topic/xref xref" href="../Origins/JDBCConsumer.html#concept_rzl_s1t_kz">Event
                            Record section</a> of the JDBC Query Consumer documentation. Every
                        event-generating stage has event details in a similar section. </p><p class="- topic/p p">The
                        query success and failure events can be useful, so you might use a Stream
                        Selector to route those records to a separate event stream. But let's say we
                        don't care about those events, we just want the no-more-data event to pass
                        to the Pipeline Finisher executor. </p></li>
                <li class="- topic/li li">Connect the event output stream to the Pipeline Finisher executor. <p class="- topic/p p">At this
                        point, all events that the origin generates come to the executor. Since the
                        JDBC Query Consumer origin generates multiple event types, this setup might
                        cause the executor to stop the pipeline too soon.</p></li>
                <li class="- topic/li li">To ensure that only the no-more-data event enters the executor, configure a
                        precondition.<p class="- topic/p p">With a precondition, only records that meet the specified
                        condition can enter the stage. </p><p class="- topic/p p">We know that each event record
                        includes the event type in the sdc.event.type record header attribute. So to
                        ensure that only no-more-data events enter the stage, we can use the
                        following expression in the
                    precondition:</p><pre class="+ topic/pre pr-d/codeblock pre codeblock"><code>${record:eventType() == 'no-more-data'}</code></pre></li>
                <li class="- topic/li li">Records that don't meet the precondition go to the stage for error handling, so
                    to avoid storing error records that we don't care about â that is, the query
                    success and failure events â let's also set the <span class="+ topic/ph ui-d/uicontrol ph uicontrol">On Record
                        Error</span> property to <span class="+ topic/ph ui-d/uicontrol ph uicontrol">Discard</span>.<p class="- topic/p p">So here's
                        the Pipeline Finisher: </p><p class="- topic/p p"><img class="- topic/image image" id="concept_kff_ykv_lz__image_ucl_4qw_lz" src="../Graphics/Event-StopPipe-Finisher.png" height="354" width="563"/></p></li>
                <li class="- topic/li li">Now, to get notified when the Pipeline Finisher stops the pipeline, configure
                    the pipeline to send an email when the pipeline state changes to Finished.
                        <div class="- topic/p p">You can use this option when <span class="- topic/ph ph">Data Collector</span> is <span class="- topic/ph ph"><a class="- topic/xref xref" href="../Configuration/DCConfig.html#concept_it1_wwg_xz">set up to send email</a></span>. You can alternatively use the pipeline state
                        notification to send a webhook, or use an <a class="- topic/xref xref" href="../Executors/Email.html#concept_sjs_sfp_qz">Email executor</a>
                        in the pipeline to send a customized email. Since we only need a simple
                        notification, let's send a basic email based on the pipeline state: <ol class="- topic/ol ol" type="a" id="concept_kff_ykv_lz__ol_wcz_wsb_yz" data-ofbid="concept_kff_ykv_lz__ol_wcz_wsb_yz">
                            <li class="- topic/li li">Click in the canvas to view the pipeline configuration, and click
                                the <span class="+ topic/keyword ui-d/wintitle keyword wintitle">Notifications</span> tab. </li>
                            <li class="- topic/li li">In the <span class="+ topic/ph ui-d/uicontrol ph uicontrol">Notify on Pipeline State Changes</span>,
                                leave the <span class="+ topic/ph ui-d/uicontrol ph uicontrol">Finished</span> state and remove the other
                                default states. </li>
                            <li class="- topic/li li">Then, enter the email addresses to receive the email:</li>
                        </ol></div><p class="- topic/p p"><img class="- topic/image image" id="concept_kff_ykv_lz__image_qmr_cpv_xz" src="../Graphics/Event-StopPipe-Notification.png" height="141" width="642"/></p></li>
            </ol>That's it!</div>
        <p class="- topic/p p">With this setup, the JDBC Query Consumer passes a no-more-data event when it completes
            processing all data returned by the query, and the Pipeline Finisher executor stops the
            pipeline and transitions the pipeline to a Finished state. All other events generated by
            the origin are discarded. <span class="- topic/ph ph">Data Collector</span>
            sends notification so you know when the pipeline finishes, and the next time you want to
            process more data, you can just start the pipeline again. </p>
    </div>
</article></article></main></div>

                        
                        
                        


                    </div>
                    
                </div>
            </div>


        </div> <nav class="navbar navbar-default wh_footer" data-whc_version="25.0">
  <div class=" footer-container  mx-auto">
    <!-- script for Data Collector, all flavors, but only used when accessed directly, not from portal --><script>
  (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
  (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
  m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
  })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');

  ga('create', 'UA-60917135-3', 'auto');
  ga('send', 'pageview');
</script>
  </div>
</nav>

        
        <div id="go2top">
            <span class="oxy-icon oxy-icon-up"></span>
        </div>
        
        <!-- The modal container for images -->
        <div id="modal_img_large" class="modal">
            <span class="close oxy-icon oxy-icon-remove"></span>
            <!-- Modal Content (The Image) -->
            <div id="modal_img_container"></div>
            <!-- Modal Caption (Image Text) -->
            <div id="caption"></div>
        </div>
        
        
        Â© 2023 StreamSets, Inc.

    </body>
</html>